#! /usr/bin/env python3
# -*- coding: utf-8 -*-
# author: maxiao
# date: 2023-03-13 18:27
# description: 抓取古诗集内容

import time

from faker import Faker
from lxml import etree
import pandas
import requests
import logging
logging.basicConfig(level=logging.INFO, format="%(asctime)s-->%(levelname)s-->%(message)s")


class Poetry:
    """实现发送请求、页面解析、内容持久化存储"""
    poem = []
    author = []
    name = []    

    def get_resource(self, page):
        """发送请求，获取响应的数据"""
        f = Faker(locale="zh_CN")
        header = {'UserAgent': f.user_agent()}
        url = f"https://www.gushiji.cc/mingju/p{page}.html"
        self.response = requests.get(url, headers=header).text

    def extract_data(self):
        """使用etree来解析源码，通过xpath提取指定数据"""
        doc = etree.HTML(self.response)
        # 此页每页末尾有一空行，即没有内容的<li></li>标签对
        li_list = doc.xpath('//ul[@class="good-mingju main-data"]/li')[:-1]
        for li in li_list:
            # 追加诗句内容
            self.poem.append(li.xpath('a[@class="name"]/text()')[0])
            # 追加作者内容（添加异常捕获）
            try:
                author_info = li.xpath('em/a/text()')[0]
            except IndexError:
                author_info = " "
            self.author.append(author_info)
            # 追加标题内容（添加异常处理）
            try:
                name_info = li.xpath('a[2]/text()')[0]
            except IndexError:
                name_info = " "
            self.name.append(name_info)

    def save_file(self, name: str = "peom.csv"):
        """保存数据至csv文件或xlsx文件"""
        df = pandas.DataFrame({
            'title': self.name,
            'author': self.author,
            'content': self.poem
        })
        if name.endswith('csv'):
            df.to_csv(name, index=False, encoding='utf-8')
        elif name.endswith('xlsx'):
            df.to_excel(name, sheet_name="名言诗句集锦", index=False)
        else:
            raise ValueError("文件格式错误，仅支持csv及xlsx格式" )


# 主程序入口
if __name__ == "__main__":
    print("本脚本未做数据校验，因此请按提示输入！！！")
    start_page = int(input("请输入起始页码（1-290）: "))
    stop_page = int(input("请输入终止页码（1-290）: "))
    # 记录程序开始运行的时间戳
    start_time = time.time()
    p_obj = Poetry()
    # 计数器
    i = 0
    for num in range(start_page, stop_page + 1):
        p_obj.get_resource(num)
        p_obj.extract_data()
        i += 1
        rate = round( 100 * i / (stop_page + 1 - start_page), 2)
        logging.info(f"第{num}页的数据已提取完成，进度{rate}%")
        time.sleep(0.2)
    p_obj.save_file('poem.xlsx')
    end_time = time.time()
    run_time = round(end_time - start_time, 2)
    print(f"本次脚本运行时长: {run_time}秒")

